import requests

from multiprocessing import Process

from lxml import etree

import time

class Run(Process):
    def __init__(self, page):
        super().__init__()

        self.str_txt_all = ''

        self.urlList = []

        self.page = page

        self.url = 'https://www.donga.com/cn/List?p0=j&p={}1&l=20&c=08'.format(self.page)

    def getUrls(self):

        rsp = requests.get(self.url)

        html = etree.HTML(rsp.text)

        self.urlList = html.xpath('/html/body/li/div/span/a/@href')

    def getRsp(self, gurl):
        try:
            rsp = requests.get(gurl)
            return rsp
        except:
            time.sleep(5)
            self.getRsp(gurl)

    def getTxt(self):

        self.getUrls()
        # //*[@id="article_text"]/text()
        num = 1
        for url in self.urlList:

            str_txt = ''

            rsp = self.getRsp(url) if num % 2 else self.getRsp(url+'?m=kor')

            html = etree.HTML(rsp.text)

            txt = html.xpath('//*[@id="article_text"]/text()')

            for i in txt:

                str_txt += i + '\n' * 2

            if num % 2 :

                self.str_txt_all += str_txt + '\n' * 3

            else:

                self.str_txt_all += str_txt + '\n' * 8

                print(self.str_txt_all)
                with open('东亚日报-文化.doc','ab') as f:
                    f.write(self.str_txt_all.encode())

                self.str_txt_all = ''

            num += 1

            time.sleep(2)

def getTxt(page_s, page_c):

    for i in range(page_c):

        p = Run(i + page_s)

        p.start()

def main():

    getTxt(1, 50)

if __name__ == '__main__':

    main()